#!/usr/local/bin/python
#-*- encoding:utf-8 -*-
from whoosh.index import create_in  
from whoosh.fields import *  
from chinesetokenizer import ChineseAnalyzer
#from whoosh.analysis import RegexAnalyzer  
#analyzer = RegexAnalyzer(ur"([\u4e00-\u9fa5])|(\w+(\.?\w+)*)")

from whoosh.index import open_dir
from whoosh.scoring import *
from whoosh.qparser import QueryParser
import ConfigParser

config = ConfigParser.ConfigParser();
config.read('config');

ix = open_dir(config.get('bm25', 'BackupIndex'));
reader = ix.reader();
searcher = ix.searcher();

dictList = [(reader.doc_frequency('content', t), searcher.idf('content', t), t) for t in reader.field_terms('content')];
dictList.sort();
MAX = len(dictList);
print MAX;
d = open(config.get('rmls', 'dictionary'),'w');
wd = open(config.get('rmls', 'wasted_dictionary'), 'w');
#docCount = searcher.doc_count();
#print docCount;
K = 8;
for pair in dictList:
	if pair[0] < K or pair[1] < 2:
		wd.write('%s\t%f\t%f\n'%(pair[2].encode('utf-8'), pair[1], pair[0]));
	else:
		d.write('%s\t%f\t%f\n' % (pair[2].encode('utf-8'), pair[1], pair[0]));

d.close();
wd.close();
